





<!DOCTYPE html>
<html class="writer-html5" lang="zh-CN" >
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>TVM runtime系统 &mdash; tvm 0.8.dev1982 文档</title>
  

  
  <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/css/bootstrap.min.css" integrity="sha384-Gn5384xqQ1aoWXA+058RXPxPg6fy4IWvTNh0E263XmFcJlSAwiGgFAW/dAiS6JXm" crossorigin="anonymous">
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/gallery.css" type="text/css" />
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/tlcpack_theme.css" type="text/css" />

  
  
    <link rel="shortcut icon" href="../_static/tvm-logo-square.png"/>
  

  
  
  
  
    
      <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
        <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
        <script src="../_static/jquery.js"></script>
        <script src="../_static/underscore.js"></script>
        <script src="../_static/doctools.js"></script>
        <script src="../_static/translations.js"></script>
    
    <script type="text/javascript" src="../_static/js/theme.js"></script>

    
    <script type="text/javascript" src="../_static/js/tlcpack_theme.js"></script>
    <link rel="index" title="索引" href="../genindex.html" />
    <link rel="search" title="搜索" href="../search.html" />
    <link rel="next" title="Vulkan Runtime" href="runtimes/vulkan.html" />
    <link rel="prev" title="Design and Architecture" href="index.html" /> 
</head>

<body class="wy-body-for-nav">

   
  <div class="wy-grid-for-nav">
    
    
<header class="header">
    <div class="innercontainer">
      <div class="headerInner d-flex justify-content-between align-items-center">
          <div class="headerLogo">
               <a href="https://tvm.apache.org/"><img src=https://tvm.apache.org/assets/images/logo.svg alt="logo"></a>
          </div>

          <div id="headMenu" class="headerNav">
            <button type="button" id="closeHeadMenu" class="navCloseBtn"><img src="../_static/img/close-icon.svg" alt="Close"></button>
             <ul class="nav">
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/community>Community</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/download>Download</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/vta>VTA</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/blog>Blog</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvm.apache.org/docs>Docs</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvmconf.org>Conference</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://github.com/apache/tvm/>Github</a>
                </li>
                <li class="nav-item">
                   <a class="nav-link" href=https://tvmchinese.github.io/declaration_zh_CN.html>About-Translators</a>
                </li>
             </ul>
               <div class="responsivetlcdropdown">
                 <button type="button" class="btn-link">
                   ASF
                 </button>
                 <ul>
                     <li>
                       <a href=https://apache.org/>Apache Homepage</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/licenses/>License</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/sponsorship.html>Sponsorship</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/security/>Security</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/thanks.html>Thanks</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/events/current-event>Events</a>
                     </li>
                     <li>
                       <a href=https://www.zhihu.com/column/c_1429578595417563136>Zhihu</a>
                     </li>
                 </ul>
               </div>
          </div>
            <div class="responsiveMenuIcon">
              <button type="button" id="menuBtn" class="btn-menu"><img src="../_static/img/menu-icon.svg" alt="Menu Icon"></button>
            </div>

            <div class="tlcDropdown">
              <div class="dropdown">
                <button type="button" class="btn-link dropdown-toggle" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">
                  ASF
                </button>
                <div class="dropdown-menu dropdown-menu-right">
                  <ul>
                     <li>
                       <a href=https://apache.org/>Apache Homepage</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/licenses/>License</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/sponsorship.html>Sponsorship</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/security/>Security</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/foundation/thanks.html>Thanks</a>
                     </li>
                     <li>
                       <a href=https://www.apache.org/events/current-event>Events</a>
                     </li>
                     <li>
                       <a href=https://www.zhihu.com/column/c_1429578595417563136>Zhihu</a>
                     </li>
                  </ul>
                </div>
              </div>
          </div>
       </div>
    </div>
 </header>
 
    <nav data-toggle="wy-nav-shift" class="wy-nav-side fixed">
      <div class="wy-side-scroll">
        <div class="wy-side-nav-search" >
          

          
            <a href="../index.html">
          

          
            
            <img src="../_static/tvm-logo-small.png" class="logo" alt="Logo"/>
          
          </a>

          
            
            
                <div class="version">
                  0.8.dev1982
                </div>
            
          

          
<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

          
        </div>

        
        <div class="wy-menu wy-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          
            
            
              
            
            
              <p class="caption" role="heading"><span class="caption-text">如何开始</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../install/index.html">安装 TVM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contribute/index.html">贡献者指南</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">用户引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../tutorial/index.html">User Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../how_to/index.html">How To Guides</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">开发者引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../dev/tutorial/index.html">Developer Tutorial</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dev/how_to/how_to.html">开发者指南</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">架构指南</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="index.html">Design and Architecture</a><ul class="current">
<li class="toctree-l2"><a class="reference internal" href="index.html#example-compilation-flow">编译流程示例</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#logical-architecture-components">逻辑架构组件</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-support">tvm/support</a></li>
<li class="toctree-l2 current"><a class="reference internal" href="index.html#tvm-runtime">tvm/runtime</a><ul class="current">
<li class="toctree-l3 current"><a class="current reference internal" href="#">TVM runtime系统</a><ul>
<li class="toctree-l4"><a class="reference internal" href="#packedfunc">PackedFunc</a></li>
<li class="toctree-l4"><a class="reference internal" href="#module">模块</a></li>
<li class="toctree-l4"><a class="reference internal" href="#remote-deployment">Remote Deployment</a></li>
<li class="toctree-l4"><a class="reference internal" href="#tvm-object-and-compiler-stack">TVM Object and Compiler Stack</a></li>
<li class="toctree-l4"><a class="reference internal" href="#implementation-details">Implementation Details</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="#runtime-specific-information">Runtime-Specific Information</a><ul>
<li class="toctree-l4"><a class="reference internal" href="runtimes/vulkan.html">Vulkan Runtime</a></li>
</ul>
</li>
<li class="toctree-l3"><a class="reference internal" href="debugger.html">Debugger</a></li>
<li class="toctree-l3"><a class="reference internal" href="virtual_machine.html">Putting the VM in TVM: The Relay Virtual Machine</a></li>
<li class="toctree-l3"><a class="reference internal" href="introduction_to_module_serialization.html">Introduction to Module Serialization</a></li>
<li class="toctree-l3"><a class="reference internal" href="device_target_interactions.html">Device/Target Interactions</a></li>
</ul>
</li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-node">tvm/node</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-ir">tvm/ir</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-target">tvm/target</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-tir">tvm/tir</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-arith">tvm/arith</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-te">tvm/te</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-topi">tvm/topi</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-relay">tvm/relay</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#tvm-autotvm">tvm/autotvm</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#frontends">Frontends</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#security">安全</a></li>
<li class="toctree-l2"><a class="reference internal" href="index.html#microtvm">microTVM</a></li>
</ul>
</li>
</ul>
<p class="caption" role="heading"><span class="caption-text">主题引导</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../topic/microtvm/index.html">microTVM：裸机使用TVM</a></li>
<li class="toctree-l1"><a class="reference internal" href="../topic/vta/index.html">VTA: Versatile Tensor Accelerator</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">参考指南</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../reference/langref/index.html">语言参考</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/api/python/index.html">Python API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/api/links.html">Other APIs</a></li>
<li class="toctree-l1"><a class="reference internal" href="../reference/publications.html">Publications</a></li>
<li class="toctree-l1"><a class="reference internal" href="../genindex.html">索引</a></li>
</ul>

            
          
        </div>
        
      </div>
    </nav>

    <section data-toggle="wy-nav-shift" class="wy-nav-content-wrap">
      
      <nav class="wy-nav-top" aria-label="top navigation" data-toggle="wy-nav-top">
        
            <div class="togglemenu">

            </div>
            <div class="nav-content">
              <!-- tvm -->
              Table of content
            </div>
        
      </nav>


      <div class="wy-nav-content">
        
        <div class="rst-content">
        

          




















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="wy-breadcrumbs">
    
      <li><a href="../index.html">Docs</a> <span class="br-arrow">></span></li>
        
          <li><a href="index.html">Design and Architecture</a> <span class="br-arrow">></span></li>
        
      <li>TVM runtime系统</li>
    
    
      <li class="wy-breadcrumbs-aside">
        
            
            <a href="../_sources/arch/runtime.rst.txt" rel="nofollow"> <img src="../_static//img/source.svg" alt="viewsource"/></a>
          
        
      </li>
    
  </ul>

  
  <hr/>
</div>
          <div role="main" class="document" itemscope="itemscope" itemtype="http://schema.org/Article">
           <div itemprop="articleBody">
            
  <div class="section" id="tvm-runtime-system">
<span id="id1"></span><h1>TVM runtime系统<a class="headerlink" href="#tvm-runtime-system" title="永久链接至标题">¶</a></h1>
<p>TVM支持多种编程语言用于编译器堆栈的开发和部署。在本文中，我们会解释TVM runtime的关键元素。</p>
<img alt="https://tvm.apache.org/images/release/tvm_flexible.png" src="https://tvm.apache.org/images/release/tvm_flexible.png" />
<p>我们需要满足许多有趣的要求：</p>
<ul class="simple">
<li><p>部署：用python/javascrip/c++语言调用编译后的函数。</p></li>
<li><p>调试：用python定义一个函数并在一个编译后的函数中调用它。</p></li>
<li><p>链接：编写驱动代码来调用设备特定代码（CUDA）并在编译后的主机函数中调用它。</p></li>
<li><p>原型：用python定义一个IR pass然后从C++后端中调用。</p></li>
<li><p>公开：从用c++开发的编译器堆栈到前端（如python）</p></li>
<li><p>实验：将编译后的函数发送到嵌入式设备来直接运行。</p></li>
</ul>
<p>我们想要能用任何语言定义一个函数然后用另一个语言来调用它。我们还希望runtime核能小到足以部署在嵌入式设备上。</p>
<div class="section" id="packedfunc">
<span id="tvm-runtime-system-packed-func"></span><h2>PackedFunc<a class="headerlink" href="#packedfunc" title="永久链接至标题">¶</a></h2>
<p>我们发现，<cite>PackedFunc`_是一种简单但优雅的解决方案，可以解决提出的挑战。单个 ``PackedFunc`</cite> 对象表示一个函数调用，这个函数的调用方和被调用方可能使用的不同语言。</p>
<p>下面代码块提供了一个C++中的例子</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="cp">#include</span> <span class="cpf">&lt;tvm/runtime/packed_func.h&gt;</span><span class="cp"></span>

<span class="kt">void</span> <span class="nf">MyAdd</span><span class="p">(</span><span class="n">TVMArgs</span> <span class="n">args</span><span class="p">,</span> <span class="n">TVMRetValue</span><span class="o">*</span> <span class="n">rv</span><span class="p">)</span> <span class="p">{</span>
  <span class="c1">// automatically convert arguments to desired type.</span>
  <span class="kt">int</span> <span class="n">a</span> <span class="o">=</span> <span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">];</span>
  <span class="kt">int</span> <span class="n">b</span> <span class="o">=</span> <span class="n">args</span><span class="p">[</span><span class="mi">1</span><span class="p">];</span>
  <span class="c1">// automatically assign value return to rv</span>
  <span class="o">*</span><span class="n">rv</span> <span class="o">=</span> <span class="n">a</span> <span class="o">+</span> <span class="n">b</span><span class="p">;</span>
<span class="p">}</span>

<span class="kt">void</span> <span class="nf">CallPacked</span><span class="p">()</span> <span class="p">{</span>
  <span class="n">PackedFunc</span> <span class="n">myadd</span> <span class="o">=</span> <span class="n">PackedFunc</span><span class="p">(</span><span class="n">MyAdd</span><span class="p">);</span>
  <span class="c1">// get back 3</span>
  <span class="kt">int</span> <span class="n">c</span> <span class="o">=</span> <span class="n">myadd</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">);</span>
<span class="p">}</span>
</pre></div>
</div>
<p>在上述代码块中，我们定义了一个PackedFunc MyAdd。 它有两个参数： <code class="docutils literal notranslate"><span class="pre">args</span></code> 表示输入参数， <code class="docutils literal notranslate"><span class="pre">rv</span></code> 表示返回值。这个函数被类型擦除，即这个函数签名不限制要传入的输入类型和要返回的类型。在后台，当我们调用一个 PackedFunc,它将输入参数打包到堆栈上的TVMArgs，然后通过TVMRetValue得到返回结果。</p>
<p>多亏于C++中的模板技巧，我们可以像调用一个正常函数一样调用一个 PackedFunc。因为他的类型擦除特性，我们可以从动态语言像python调用一个 PackedFunc，而无需为每一个新类型函数的生成添加额外的粘合代码。下面的示例用C++注册了 PackedFunc然后用python来调用它。</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="c1">// register a global packed function in c++</span>
<span class="n">TVM_REGISTER_GLOBAL</span><span class="p">(</span><span class="s">&quot;myadd&quot;</span><span class="p">)</span>
<span class="p">.</span><span class="n">set_body</span><span class="p">(</span><span class="n">MyAdd</span><span class="p">);</span>
</pre></div>
</div>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">tvm</span>

<span class="n">myadd</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">get_global_func</span><span class="p">(</span><span class="s2">&quot;myadd&quot;</span><span class="p">)</span>
<span class="c1"># prints 3</span>
<span class="k">print</span><span class="p">(</span><span class="n">myadd</span><span class="p">(</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">))</span>
</pre></div>
</div>
<p>PackedFunc的大部分魔力在于``TVMArgs`` 和 <a href="#id1"><span class="problematic" id="id2">``</span></a>TVMRetValue``的结构。我们限制了一系列可能会被传输的类型。以下是常见的：</p>
<ul class="simple">
<li><p>int, float 和 string</p></li>
<li><p>PackedFunc 本身</p></li>
<li><p>编译模块的模块</p></li>
<li><p>用于张量对象交换的DLTensor*</p></li>
<li><p>表示任意IR对象的TVM对象</p></li>
</ul>
<p>该限制使得实现简单，无需序列化。尽管PackedFunc是最小的，但对于深度学习部署的用例来说已经足够了，因为大多数函数只接受DLTensor或数字。</p>
<p>既然一个PackedFunc可以将另一个PackedFunc当作输入，我们可以把函数（作为PackedFunc）从python传给C++。</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="n">TVM_REGISTER_GLOBAL</span><span class="p">(</span><span class="s">&quot;callhello&quot;</span><span class="p">)</span>
<span class="p">.</span><span class="n">set_body</span><span class="p">([](</span><span class="n">TVMArgs</span> <span class="n">args</span><span class="p">,</span> <span class="n">TVMRetValue</span><span class="o">*</span> <span class="n">rv</span><span class="p">)</span> <span class="p">{</span>
  <span class="n">PackedFunc</span> <span class="n">f</span> <span class="o">=</span> <span class="n">args</span><span class="p">[</span><span class="mi">0</span><span class="p">];</span>
  <span class="n">f</span><span class="p">(</span><span class="s">&quot;hello world&quot;</span><span class="p">);</span>
<span class="p">});</span>
</pre></div>
</div>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">tvm</span>

<span class="k">def</span> <span class="nf">callback</span><span class="p">(</span><span class="n">msg</span><span class="p">):</span>
  <span class="k">print</span><span class="p">(</span><span class="n">msg</span><span class="p">)</span>

<span class="c1"># convert to PackedFunc</span>
<span class="n">f</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">convert</span><span class="p">(</span><span class="n">callback</span><span class="p">)</span>
<span class="n">callhello</span> <span class="o">=</span> <span class="n">tvm</span><span class="o">.</span><span class="n">get_global_func</span><span class="p">(</span><span class="s2">&quot;callhello&quot;</span><span class="p">)</span>
<span class="c1"># prints hello world</span>
<span class="n">callhello</span><span class="p">(</span><span class="n">f</span><span class="p">)</span>
</pre></div>
</div>
<p>TVM提供一个允许我们将PackedFunc嵌入到任何语言的`最小 C API`_。除了python，目前我们还支持 <a class="reference external" href="https://github.com/apache/tvm/tree/main/jvm">java</a> 和 <a class="reference external" href="https://github.com/apache/tvm/tree/main/web">javascript</a>。这种嵌入式API的概念非常像Lua，只是我们没有用新的语言而是使用C++。</p>
<p>一个关于PackedFunc的有趣的事实是，我们在编译器和部署堆栈都用到了它。</p>
<ul class="simple">
<li><p>TVM所有的编译pass函数都作为PackedFunc公开给了前端，</p></li>
<li><p>编译后的模块也将作为 PackedFunc的编译后函数返回。</p></li>
</ul>
<p>为了保持runtime的最小化，我们将IR对象支持与部署runtime分离。根据包含的驱动模块的数量（如CUDA），生成的runtime大概占200-600K。</p>
<p>与普通函数相比，调用PackedFunc的开销很小，因为它只在堆栈上保存了一些值。因此，只要我们不包装小函数就可以了。总之，PackedFunc是TVM中的通用粘合剂，我们在TVM中广泛使用它来支持编译器和部署。</p>
</div>
<div class="section" id="module">
<span id="tvm-runtime-system-module"></span><h2>模块<a class="headerlink" href="#module" title="永久链接至标题">¶</a></h2>
<p>由于TVM支持多种类型的设备，我们需要支持不同类型的驱动程序。我们必须使用驱动程序API加载内核，以压缩格式设置参数并启动内核。我们还需要修补驱动程序API，以便公开的函数是线程安全的。因此，我们经常需要在C++中实现这些驱动程序粘合，并将它们公开给用户。我们当然不能对每种类型的函数都这样做，所以PackedFunc仍是我们的答案。</p>
<p>TVM将编译对象定义为 <a class="reference internal" href="#module">模块</a>。用户就可以从模块中以PackedFunc的形式得到编译后的函数。生成的编译代码可以在runtime中动态得从模块中获得函数。它在第一次调用中缓存函数句柄，并在后续调用中再次使用。我们使用它链接设备代码并用生成的代码调回到到任意PackedFunc（例如python）中。</p>
<p>ModuleNode 是一个抽象类，可以被每种设备实现。到目前为止，我们支持CUDA、Metal、OpenCL和加载动态共享库的模块。这种抽象易于引入新设备，并且我们不需要为每种类型的设备重新生成主机代码。</p>
</div>
<div class="section" id="remote-deployment">
<h2>Remote Deployment<a class="headerlink" href="#remote-deployment" title="永久链接至标题">¶</a></h2>
<p>The PackedFunc and Module system also makes it easy to ship the function into remote devices directly.
Under the hood, we have an RPCModule that serializes the arguments to do the data movement and launches the computation on the remote.</p>
<img alt="https://tvm.apache.org/images/release/tvm_rpc.png" src="https://tvm.apache.org/images/release/tvm_rpc.png" />
<p>The RPC server itself is minimum and can be bundled into the runtime. We can start a minimum TVM
RPC server on iPhone/android/raspberry pi or even the browser. The cross compilation on server and shipping of the module for testing can be done in the same script. Checkout
<a class="reference internal" href="../tutorial/cross_compilation_and_rpc.html#tutorial-cross-compilation-and-rpc"><span class="std std-ref">Cross Compilation and RPC</span></a> for more details.</p>
<p>This instant feedback gives us a lot of advantages. For example, to test the correctness of generated code on iPhone, we no longer have to write test-cases in swift/objective-c from scratch – We can use RPC to execute on iPhone, copy the result back and do verification on the host via numpy. We can also do the profiling using the same script.</p>
</div>
<div class="section" id="tvm-object-and-compiler-stack">
<h2>TVM Object and Compiler Stack<a class="headerlink" href="#tvm-object-and-compiler-stack" title="永久链接至标题">¶</a></h2>
<p>As we mentioned earlier, we build compiler stack API on top of the PackedFunc runtime system.
We faced a constant changing of the compiler API for the need of research. We need a new language object or IR node whenever we want to test out new primitives.
However, we don’t want to change our API from time to time. Besides that, we also want to</p>
<ul class="simple">
<li><p>be able to serialize any language object and IRs</p></li>
<li><p>be able to explore, print, and manipulate the IR objects in front-end language to do quick prototyping.</p></li>
</ul>
<p>We introduced a base class, called <a class="reference external" href="https://github.com/apache/tvm/blob/main/include/tvm/runtime/object.h">Object</a> to solve this problem.
All the language object in the compiler stack is a subclass of <code class="docutils literal notranslate"><span class="pre">Object</span></code>. Each object contains a string type_key that uniquely identifies
the type of object. We choose string instead of int as type key so new <code class="docutils literal notranslate"><span class="pre">Object</span></code> class can be added in the decentralized fashion without
adding the code back to the central repo. To ease the speed of dispatching, we allocate an integer type_index at runtime for each type_key.</p>
<p>Since usually one <code class="docutils literal notranslate"><span class="pre">Object</span></code> could be referenced in multiple places in the language, we use a shared_ptr to keep
track of reference. We use <code class="docutils literal notranslate"><span class="pre">ObjectRef</span></code> class to represent a reference to the <code class="docutils literal notranslate"><span class="pre">Object</span></code>.
We can roughly view <code class="docutils literal notranslate"><span class="pre">ObjectRef</span></code> class as shared_ptr to the <code class="docutils literal notranslate"><span class="pre">Object</span></code> container.
We can also define subclass <code class="docutils literal notranslate"><span class="pre">ObjectRef</span></code> to hold each subtypes of <code class="docutils literal notranslate"><span class="pre">Object</span></code>. Each subclass of <code class="docutils literal notranslate"><span class="pre">Object</span></code> needs to define the VisitAttr function.</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="n">class</span> <span class="n">AttrVisitor</span> <span class="p">{</span>
<span class="nl">public</span><span class="p">:</span>
  <span class="n">virtual</span> <span class="kt">void</span> <span class="n">Visit</span><span class="p">(</span><span class="k">const</span> <span class="kt">char</span><span class="o">*</span> <span class="n">key</span><span class="p">,</span> <span class="kt">double</span><span class="o">*</span> <span class="n">value</span><span class="p">)</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>
  <span class="n">virtual</span> <span class="kt">void</span> <span class="nf">Visit</span><span class="p">(</span><span class="k">const</span> <span class="kt">char</span><span class="o">*</span> <span class="n">key</span><span class="p">,</span> <span class="kt">int64_t</span><span class="o">*</span> <span class="n">value</span><span class="p">)</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>
  <span class="n">virtual</span> <span class="kt">void</span> <span class="nf">Visit</span><span class="p">(</span><span class="k">const</span> <span class="kt">char</span><span class="o">*</span> <span class="n">key</span><span class="p">,</span> <span class="kt">uint64_t</span><span class="o">*</span> <span class="n">value</span><span class="p">)</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>
  <span class="n">virtual</span> <span class="kt">void</span> <span class="nf">Visit</span><span class="p">(</span><span class="k">const</span> <span class="kt">char</span><span class="o">*</span> <span class="n">key</span><span class="p">,</span> <span class="kt">int</span><span class="o">*</span> <span class="n">value</span><span class="p">)</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>
  <span class="n">virtual</span> <span class="kt">void</span> <span class="nf">Visit</span><span class="p">(</span><span class="k">const</span> <span class="kt">char</span><span class="o">*</span> <span class="n">key</span><span class="p">,</span> <span class="kt">bool</span><span class="o">*</span> <span class="n">value</span><span class="p">)</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>
  <span class="n">virtual</span> <span class="kt">void</span> <span class="nf">Visit</span><span class="p">(</span><span class="k">const</span> <span class="kt">char</span><span class="o">*</span> <span class="n">key</span><span class="p">,</span> <span class="n">std</span><span class="o">::</span><span class="n">string</span><span class="o">*</span> <span class="n">value</span><span class="p">)</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>
  <span class="n">virtual</span> <span class="kt">void</span> <span class="nf">Visit</span><span class="p">(</span><span class="k">const</span> <span class="kt">char</span><span class="o">*</span> <span class="n">key</span><span class="p">,</span> <span class="kt">void</span><span class="o">**</span> <span class="n">value</span><span class="p">)</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>
  <span class="n">virtual</span> <span class="kt">void</span> <span class="nf">Visit</span><span class="p">(</span><span class="k">const</span> <span class="kt">char</span><span class="o">*</span> <span class="n">key</span><span class="p">,</span> <span class="n">Type</span><span class="o">*</span> <span class="n">value</span><span class="p">)</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>
  <span class="n">virtual</span> <span class="kt">void</span> <span class="nf">Visit</span><span class="p">(</span><span class="k">const</span> <span class="kt">char</span><span class="o">*</span> <span class="n">key</span><span class="p">,</span> <span class="n">ObjectRef</span><span class="o">*</span> <span class="n">value</span><span class="p">)</span> <span class="o">=</span> <span class="mi">0</span><span class="p">;</span>
  <span class="c1">// ...</span>
<span class="p">};</span>

<span class="n">class</span> <span class="nl">BaseAttrsNode</span> <span class="p">:</span> <span class="n">public</span> <span class="n">Object</span> <span class="p">{</span>
<span class="nl">public</span><span class="p">:</span>
  <span class="n">virtual</span> <span class="kt">void</span> <span class="n">VisitAttrs</span><span class="p">(</span><span class="n">AttrVisitor</span><span class="o">*</span> <span class="n">v</span><span class="p">)</span> <span class="p">{}</span>
  <span class="c1">// ...</span>
<span class="p">};</span>
</pre></div>
</div>
<p>Each <code class="docutils literal notranslate"><span class="pre">Object</span></code> subclass will override this to visit its members. Here is an example implementation of TensorNode.</p>
<div class="highlight-c notranslate"><div class="highlight"><pre><span></span><span class="n">class</span> <span class="nl">TensorNode</span> <span class="p">:</span> <span class="n">public</span> <span class="n">Object</span> <span class="p">{</span>
<span class="nl">public</span><span class="p">:</span>
  <span class="cm">/*! \brief The shape of the tensor */</span>
  <span class="n">Array</span><span class="o">&lt;</span><span class="n">Expr</span><span class="o">&gt;</span> <span class="n">shape</span><span class="p">;</span>
  <span class="cm">/*! \brief data type in the content of the tensor */</span>
  <span class="n">Type</span> <span class="n">dtype</span><span class="p">;</span>
  <span class="cm">/*! \brief the source operation, can be None */</span>
  <span class="n">Operation</span> <span class="n">op</span><span class="p">;</span>
  <span class="cm">/*! \brief the output index from source operation */</span>
  <span class="kt">int</span> <span class="n">value_index</span><span class="p">{</span><span class="mi">0</span><span class="p">};</span>
  <span class="cm">/*! \brief constructor */</span>
  <span class="n">TensorNode</span><span class="p">()</span> <span class="p">{}</span>

  <span class="kt">void</span> <span class="n">VisitAttrs</span><span class="p">(</span><span class="n">AttrVisitor</span><span class="o">*</span> <span class="n">v</span><span class="p">)</span> <span class="n">final</span> <span class="p">{</span>
    <span class="n">v</span><span class="o">-&gt;</span><span class="n">Visit</span><span class="p">(</span><span class="s">&quot;shape&quot;</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">shape</span><span class="p">);</span>
    <span class="n">v</span><span class="o">-&gt;</span><span class="n">Visit</span><span class="p">(</span><span class="s">&quot;dtype&quot;</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">dtype</span><span class="p">);</span>
    <span class="n">v</span><span class="o">-&gt;</span><span class="n">Visit</span><span class="p">(</span><span class="s">&quot;op&quot;</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">op</span><span class="p">);</span>
    <span class="n">v</span><span class="o">-&gt;</span><span class="n">Visit</span><span class="p">(</span><span class="s">&quot;value_index&quot;</span><span class="p">,</span> <span class="o">&amp;</span><span class="n">value_index</span><span class="p">);</span>
  <span class="p">}</span>
<span class="p">};</span>
</pre></div>
</div>
<p>In the above examples, both <code class="docutils literal notranslate"><span class="pre">Operation</span></code> and <code class="docutils literal notranslate"><span class="pre">Array&lt;Expr&gt;</span></code> are ObjectRef.
The VisitAttrs gives us a reflection API to visit each member of the object.
We can use this function to visit the node and serialize any language object recursively.
It also allows us to get members of an object easily in front-end language.
For example, in the following code, we accessed the op field of the TensorNode.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">tvm</span>
<span class="kn">from</span> <span class="nn">tvm</span> <span class="kn">import</span> <span class="n">te</span>

<span class="n">x</span> <span class="o">=</span> <span class="n">te</span><span class="o">.</span><span class="n">placeholder</span><span class="p">((</span><span class="mi">3</span><span class="p">,</span><span class="mi">4</span><span class="p">),</span> <span class="n">name</span><span class="o">=</span><span class="s2">&quot;x&quot;</span><span class="p">)</span>
<span class="c1"># access the op field of TensorNode</span>
<span class="k">print</span><span class="p">(</span><span class="n">x</span><span class="o">.</span><span class="n">op</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
</pre></div>
</div>
<p>New <code class="docutils literal notranslate"><span class="pre">Object</span></code> can be added to C++ without changing the front-end runtime, making it easy to make extensions to the compiler stack.
Note that this is not the fastest way to expose members to front-end language, but might be one of the simplest
approaches possible. We also find that it fits our purposes as we mainly use python for testing and prototyping and still use c++
to do the heavy lifting job.</p>
</div>
<div class="section" id="implementation-details">
<h2>Implementation Details<a class="headerlink" href="#implementation-details" title="永久链接至标题">¶</a></h2>
<p>Each argument in PackedFunc contains a union value <a class="reference external" href="https://github.com/apache/tvm/blob/main/include/tvm/runtime/c_runtime_api.h#L135">TVMValue</a>
and a type code. This design allows the dynamically typed language to convert to the corresponding type directly, and statically typed language to
do runtime type checking during conversion.</p>
<p>The relevant files are</p>
<ul class="simple">
<li><p><a class="reference external" href="https://github.com/apache/tvm/blob/main/include/tvm/runtime/packed_func.h">packed_func.h</a> for C++ API</p></li>
<li><p><a class="reference external" href="https://github.com/apache/tvm/blob/main/src/runtime/c_runtime_api.cc#L262">c_runtime_api.cc</a> for C API and how to provide callback.</p></li>
</ul>
<p>To support extension types, we used a registry system to register type related information, like support of any
in C++, see <a class="reference external" href="https://github.com/apache/tvm/tree/main/apps/extension">Extension types</a> for more details.</p>
</div>
</div>
<div class="section" id="runtime-specific-information">
<h1>Runtime-Specific Information<a class="headerlink" href="#runtime-specific-information" title="永久链接至标题">¶</a></h1>
<div class="toctree-wrapper compound">
<ul>
<li class="toctree-l1"><a class="reference internal" href="runtimes/vulkan.html">Vulkan Runtime</a></li>
</ul>
</div>
</div>


           </div>
           
          </div>
          

<footer>

    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="runtimes/vulkan.html" class="btn btn-neutral float-right" title="Vulkan Runtime" accesskey="n" rel="next">下一个 <span class="fa fa-arrow-circle-right"></span></a>
      
      
        <a href="index.html" class="btn btn-neutral float-left" title="Design and Architecture" accesskey="p" rel="prev"><span class="fa fa-arrow-circle-left"></span> 上一个</a>
      
    </div>

<div id="button" class="backtop"><img src="../_static//img/right.svg" alt="backtop"/> </div>
<section class="footerSec">
    <div class="footerHeader">
      <ul class="d-flex align-md-items-center justify-content-between flex-column flex-md-row">
        <li class="copywrite d-flex align-items-center">
          <h5 id="copy-right-info">© 2020 Apache Software Foundation | All right reserved</h5>
        </li>
      </ul>

    </div>

    <ul>
      <li class="footernote">Copyright © 2020 The Apache Software Foundation. Apache TVM, Apache, the Apache feather, and the Apache TVM project logo are either trademarks or registered trademarks of the Apache Software Foundation.</li>
    </ul>

</section>
</footer>
        </div>
      </div>

    </section>

  </div>
  

    <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.12.9/umd/popper.min.js" integrity="sha384-ApNbgh9B+Y1QKtv3Rn7W3mgPxhU9K/ScQsAP7hUibX39j7fakFPskvXusvfa0b4Q" crossorigin="anonymous"></script>
    <script src="https://maxcdn.bootstrapcdn.com/bootstrap/4.0.0/js/bootstrap.min.js" integrity="sha384-JZR6Spejh4U02d8jOt6vLEHfe/JQGiRRSQQxSfFWpi1MquVdAyjUar5+76PVCmYl" crossorigin="anonymous"></script>

  </body>
  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>

  
  
    
    <!-- Theme Analytics -->
    <script>
    (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
      (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
      m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
    })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

    ga('create', 'UA-75982049-2', 'auto');
    ga('send', 'pageview');
    </script>

    
   

</body>
</html>